from requests.exceptions import RequestException
import requests
import re,json,os

def writeFile(content):
    '''将爬取的内容写入文件'''
    with open('./maoyan.txt','a',encoding='utf-8') as f :
        f.write(json.dumps(content,ensure_ascii=False) + "\n")
        f.close()

def getpage(offset):
    '''爬取每一页的内容'''
    url = 'https://maoyan.com/board/4?offset='+str(offset)
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"}

    try:
        res = requests.get(url,headers=headers)
        if res.status_code == 200 :
            html = res.content.decode('utf-8')
            # print(html[:5000])
            pat = '<dd>.*?board-index.*?">(\d+)</i>.*?title="(.*?)".*?<img data-src="(.*?)".*?"star">(.*?)</p>.*?"releasetime">(.*?)</p>.*?"integer">(.*?)</i>.*?"fraction">(.*?)</i>.*?</dd>'

            pattern = re.compile(pat,re.S)
            dlist = pattern.findall(str(html))
            x = offset+1
            for v in dlist:
                yield{
                    '|名次：':v[0],
                    '|名称:':v[1].strip(),
                    '|':v[3].strip(),
                    '|':v[4],
                    '|评分：':v[5]+v[6],
                    '|图片：':v[2],
                }
                img = requests.get(v[2])
                with open('./images/maoyan%s.jpg' % x,'wb') as f :
                    f.write(img.content)
                    f.close()
                x+=1
            print('第%s页完成。'%int(offset/10+1))
                # print('|名次：'+v[0]+'|名称:'+v[1].strip()+'|'+v[3].strip()+'|'+v[4]+'|评分：'+v[5]+v[6]+'|图片：'+v[2])
        else:
            return None
    except RequestException :
        return None

def main(offset):
    '''主函数'''
    #判断当前路径是否存在images文件夹，没有则创建
    if not os.path.exists('./images'):
        os.mkdir('./images')
    for item in getpage(offset):
        writeFile(item)

if __name__ == '__main__':
    for i in range(10):
        main(i*10)